﻿using System;
using System.Collections;
using System.Configuration;
using System.Data;
using System.Data.SQLite;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using NSoup.Select;
using NSoup.Nodes;
using System.Text;
using System.Xml;
using System.Collections.Generic;
using Com.Toook.AutoDownloadHtml.DownloadHtml.ParseObj;
using RuleObj = Com.Toook.AutoDownloadHtml.DownloadHtml.ParseObj.Rule;
using System.Collections.Specialized;
using Com.Toook.AutoDownloadHtml.Util;

public partial class Index : System.Web.UI.Page
{
    protected void Page_Load(object sender, EventArgs e)
    {
        //Elements initElements = GetInitElements();
        //StepList initStepList = GetInitStepList();
        //ParseStepByStep(initElements, initStepList);

        //GetTest();

        //GetStepListFromXml();

        //ParseXml(); //news OK
        //ParseXmlMultiPage(); //parse movie multiPart OK
    }
    protected void ButtonGetContent_Click(object sender, EventArgs e)
    {
        string path = Server.MapPath("~\\App_Data\\NewsDetailXml\\NewsDetail_163.xml");//Server.MapPath("~\\App_Data\\StepNodes.xml");
        ParseXmlToStep parseStep = new ParseXmlToStep(path);
        List<RuleObj> ruleList = parseStep.GetRuleListByKindId("Ent_163");//FinanceSohu,SportSohu,Ent163_2
        string url = parseStep.KindSourceUrl;//配置文件中分类页面的原始url
        Step initStep = parseStep.InitStep;//根元素解析步骤
        int minTitleLength = parseStep.MinLinkTextLength;

        StringBuilder sb = new StringBuilder();

        //获取分类中的详细页面URL
        GetLinksInKindPage html = new GetLinksInKindPage();
        Elements initElements = html.GetInitRootElements(url, initStep);
        List<LeafResult> leafResult = html.GetPageDetail(ruleList, initElements, minTitleLength);

        foreach (LeafResult item in leafResult)
        {
            sb.Append(item.Description).Append("<br />");            
        }
        showText.Text = sb.ToString();
    }
    protected void ButtonParse_Click(object sender, EventArgs e)
    {
        string siteKindId = null;
        string url = null;
        string xmlPath = null;
        string parseType = null;
        int intervalHour = 1;
        DateTime expiryTime;
        //string connectionString = Server.MapPath("~\\App_Data\\SQLiteDatabase.db");
        string sql = "select SiteKindId, Url, XmlPath, ExpiryTime, IntervalHour, ParseType from AutoUpdateSiteExpiryTime order by ExpiryTime asc limit 1 ";
        SQLiteConnection con = SqliteHelper.GetConnection("SQLiteDatabase");//new SQLiteConnection(connectionString);
       
        SQLiteDataReader reader = null;
        SQLiteCommand cmd = new SQLiteCommand(sql, con);
        try
        {
            con.Open();
            reader = cmd.ExecuteReader();
            if (reader.Read())
            {
                siteKindId = reader["SiteKindId"].ToString();
                url = reader["Url"].ToString();
                xmlPath = reader["XmlPath"].ToString();
                expiryTime = reader.GetDateTime(3);
                intervalHour = int.Parse(reader["IntervalHour"].ToString());
                parseType = reader["ParseType"].ToString();
            }
        }
        catch (Exception ex)
        {
            LogHelper.Log.Error("读取更新时间错误", ex);
        }
        finally
        {
            SqliteHelper.CloseDbResource(con, reader);
        }
        //开始根据配置文件解析页面
        if (siteKindId != null)
        {
            if (parseType.ToLower().Equals("one"))
            {
                ParseXml(xmlPath, siteKindId);
            }
            else if (parseType.ToLower().Equals("multi"))
            {
                ParseXmlMultiPage(xmlPath, siteKindId);
            }
            else
            {
                throw new Exception("AutoUpdateSiteExpiryTime表ParseType类型错误：" + parseType);
            }
        }

        //更新数据库时间 HH是24小时制，hh是12小时制
        string expiryNew = DateTime.Now.AddHours(intervalHour).ToString("yyyy-MM-dd HH:mm:ss");
        sql = "update AutoUpdateSiteExpiryTime set ExpiryTime='" + expiryNew + "' where SiteKindId=@SiteKindId";
        //siteKindId;
        try
        {
            con.Open();
            cmd.CommandText = sql;
            cmd.Parameters.Add("@SiteKindId", DbType.AnsiString, 10).Value = siteKindId;
            cmd.ExecuteNonQuery();
        }
        catch (Exception ex)
        {
            LogHelper.Log.Error("更新AutoUpdateSiteExpiryTime表到期时间错误。", ex);
        }
        finally
        {
            con.Close();
        }
    }
    private void ParseXml(string xmlPath, string siteKindId)
    {
        //siteXml, kindId, expiryTime, interval
        string path = Server.MapPath("~\\App_Data\\" + xmlPath);//Server.MapPath("~\\App_Data\\StepNodes.xml");
        ParseXmlToStep parseStep = new ParseXmlToStep(path);
        List<RuleObj> ruleList = parseStep.GetRuleListByKindId(siteKindId);//FinanceSohu,SportSohu,Ent163_2
        string url = parseStep.KindSourceUrl;//配置文件中分类页面的原始url
        Step initStep = parseStep.InitStep;//根元素解析步骤
        int minTitleLength = parseStep.MinLinkTextLength;

        StringBuilder sb = new StringBuilder();


        //获取分类中的详细页面URL
        GetLinksInKindPage html = new GetLinksInKindPage();
        Elements initElements = html.GetInitRootElements(url, initStep);
        List<LeafResult> leafResult = html.GetByRuleList(ruleList, initElements, minTitleLength);

        StringBuilder sqlInsert = new StringBuilder();
        SQLiteConnection con = SqliteHelper.GetConnection("SQLiteDatabase");
        SQLiteCommand cmd = new SQLiteCommand(con);
        int i = 0;
        foreach (LeafResult item in leafResult)
        {
            sb.Append(item.LinkUrl).Append(", ").Append(item.LinkText).Append("<br />");
            //Sqlite 必须用insert into, 只用insert会报错
            sqlInsert.Append("Insert Into NewsUrlTemp (sourceUrl, title, siteKindId) ");
            sqlInsert.Append(" select @Url").Append(i.ToString()).Append(", @Title").Append(i.ToString()).Append(", @SiteId").Append(i.ToString());
            sqlInsert.Append(" where not exists(select * from NewsUrlTemp where sourceUrl = @Url").Append(i.ToString()).Append("); ");

            cmd.Parameters.Add("@Url" + i.ToString(), DbType.AnsiString, 200).Value = item.LinkUrl;
            cmd.Parameters.Add("@Title" + i.ToString(), DbType.String, 40).Value = item.LinkText;
            cmd.Parameters.Add("@SiteId" + i.ToString(), DbType.AnsiString, 10).Value = siteKindId;
            i++;
        }
        cmd.CommandText = sqlInsert.ToString();
        try
        {
            con.Open();
            cmd.ExecuteNonQuery();
        }
        catch (Exception ex)
        {
            LogHelper.Log.Error("插入NewsUrlTemp表错误", ex);
        }
        finally
        {
            con.Close();
        }
        showText.Text = sb.ToString();
    }
    private void ParseXmlMultiPage(string xmlPath, string siteKindId)
    {
        //siteXml, kindId, expiryTime, interval
        string path = Server.MapPath("~\\App_Data\\" + xmlPath);//Server.MapPath("~\\App_Data\\MovieKindXml\\XiaoDiaoKind.xml");
        ParseXmlToStep parseStep = new ParseXmlToStep(path);
        List<RuleObj> ruleList = parseStep.GetRuleListByKindId(siteKindId);//"XiaoDiaoDZ22"
        string url = parseStep.KindSourceUrl;//配置文件中分类页面的原始url
        Step initStep = parseStep.InitStep;//根元素解析步骤
        int minTitleLength = parseStep.MinLinkTextLength;

        StringBuilder sb = new StringBuilder();
        if (parseStep.PageRule != null)
        {
            StringBuilder ss = new StringBuilder();
            foreach (string  specialUrl in parseStep.PageRule.SpecialUrl)
            {
                ss = GetDetailUrlString(ruleList, specialUrl, initStep, minTitleLength);
                sb.Append(ss.ToString());                
            }
            for (int i = parseStep.PageRule.StartNum; i <= parseStep.PageRule.MaxNum; i++)
            {
                string orderUrl = parseStep.PageRule.BaseUrl + parseStep.PageRule.StartStr + i.ToString() + parseStep.PageRule.EndStr;
                ss = GetDetailUrlString(ruleList, orderUrl, initStep, minTitleLength);
                sb.Append(ss.ToString());
            }
        }
        else
        {
            //获取分类中的详细页面URL
            sb = GetDetailUrlString(ruleList, url, initStep, minTitleLength);
        }
        showText.Text = sb.ToString();
    }

    private static StringBuilder GetDetailUrlString(List<RuleObj> ruleList, string url, Step initStep, int minTitleLength)
    {
        //重置操作步骤到第一步之前
        foreach (RuleObj  rule in ruleList)
        {
            rule.ResetNextStep();
            rule.ResetNextLeaf();
        }

        StringBuilder sb = new StringBuilder();
        GetLinksInKindPage html = new GetLinksInKindPage();
        Elements initElements = html.GetInitRootElements(url, initStep);        
        List<LeafResult> leafResult = html.GetByRuleList(ruleList, initElements, minTitleLength);

        foreach (LeafResult item in leafResult)
        {
            sb.Append(item.LinkUrl).Append(", ").Append(item.LinkText).Append(", ").Append(item.ImgUrl).Append("<br />");
        }
        
        return sb;
    }
    //public Elements GetInitElements(string kindPageUrl, string getInitElementsByTag)
    //{
    //    Elements eles = new Elements();
    //    doc = NSoup.NSoupClient.Connect(kindPageUrl).Get();
    //    eles = doc.GetElementsByAttributeValueStarting("id", "content");
    //    return eles;
    //}
    private void GetTest()
    {
        Elements eles = new Elements();
        doc = NSoup.NSoupClient.Connect("http://sports.sohu.com/").Get();
        eles = doc.GetElementsByAttributeValueStarting("class", "center_news");
        StringBuilder sb = new StringBuilder();
        foreach (Element e in eles)
        {
            sb.Append(e.Html()).Append("<br/><br/><hr/><br/><br/>");
            e.GetElementsMatchingText("regex");
            e.GetElementsMatchingOwnText("regex");
            e.GetElementsContainingText("");
            e.GetElementsContainingOwnText("");
        }
        showText.Text = sb.ToString();
    }
    /*   #region
 
    private void ParseStepByStep(Elements initElements, StepList stepList)
    {
        StringBuilder sb = new StringBuilder();

        //i < initElements.Count 主要是判断 GetBy 返回的initElements新结果集是否有数据，没有就跳出循环
        for (int i = 0; stepList.Next() && i < initElements.Count; i++)
        {
            //Elements eles = GetBy(initElements, stepList.currentStep);
            //initElements = eles;
            
            initElements = GetBy(initElements, stepList.currentStep);
            //GetBy会遍历入参initElements，合并每个节点的结果后返回，将新结果赋给initElements，然后进行下一步操作。
            //stepList.Next()判断是否还有下一步。 
        }

        //Leaf
        Elements results = new Elements();       
        foreach (Element e in initElements)
        {
            results = e.GetElementsByTag("a");

            foreach (Element item in results)
            {
                sb.Append("title=" + item.Text() + ", link=" + item.AbsUrl("href") + "<br/>");
            }
        }

        showText.Text = sb.ToString();
    }
    /// <summary>
    /// 遍历elements节点，获取每个节点中符合step条件的子节点集合；合并所有子节点集合，作为最终的results结果集
    /// </summary>
    /// <param name="elements"></param>
    /// <param name="step"></param>
    /// <returns></returns>
    private Elements GetBy(Elements elements, Step step)
    {
        Elements results = new Elements();
        if (step.XmlResult == "E")
        {
            foreach (Element e in elements)
            {
                Element ele = e.GetElementById(step.AttrName);
                if (ele != null && ele.ChildNodes.Count > 0)
                {
                    results.Add(ele);
                }
            }
        }
        else
        {
            foreach (Element e in elements)
            {
                Elements eles = e.GetElementsByClass(step.AttrName);
                foreach (Element r in eles)
                {
                    if (r.ChildNodes.Count > 0)
                    {
                        results.Add(r);
                    }
                }
            }            
        }

        return results;
    }

    private Elements GetAuto(Element element, Step step)
    {
        Elements results = new Elements();
        switch (step.GetBy)
        {
            case ByType.Class:
                results = element.GetElementsByClass(step.AttrName);
                break;
            case ByType.Tag:
                results = element.GetElementsByTag(step.AttrName);
                break;
            case ByType.Attr:
                results = element.GetElementsByAttribute(step.AttrName);
                break;
            case ByType.IndexEquals:
                results = element.GetElementsByIndexEquals(2);
                break;
            case ByType.IndexGreaterThan:
                results = element.GetElementsByIndexGreaterThan(2);
                break;
            case ByType.IndexLessThan:
                results = element.GetElementsByIndexLessThan(2);                
                break;
            case ByType.ContainText:
                results = element.GetElementsContainingText(step.AttrName);
                break;
            default:
                break;
        }

        return results;
    }
    //
    private StepList GetInitStepList()
    {
        Step step1 = new Step();
        step1.XmlResult = "ES";
        step1.AttrName = "nav1"; // "navlidiv";

        Step step2 = new Step();
        step2.XmlResult = "ES"; // "E"
        step2.AttrName = "navli"; // "oddbg";

        StepList stepList = new StepList();
        stepList.Add(step1);
        stepList.Add(step2);

        return stepList;
    }
    private void GetStepListFromXml()
    {
        
        XmlDocument docXml = new XmlDocument();
        docXml.Load(Server.MapPath("~\\App_Data\\StepNodes.xml"));
        XmlElement root = docXml.DocumentElement;

        XmlNodeList kinds = root.SelectNodes("kind");
        //根据网站分类数初始化kindList
        List<List<StepList>> kindList = new List<List<StepList>>(kinds.Count);

        //获取各个分类
        foreach (XmlNode kind in kinds)
        {
            //一个分类可能有多个规则rule
            XmlNodeList rules = kind.SelectNodes("rule");
            List<StepList> ruleList = ParseRules(rules);
            kindList.Add(ruleList);            
        }

        ShowInfo(kindList);
    }
    private void ShowInfo(List<List<StepList>> kindList)
    {
        StringBuilder sb = new StringBuilder();
        foreach (List<StepList> rules in kindList)
        {
            sb.Append("Rule: <br/>");
            foreach (StepList stepList in rules)
            {
                sb.Append("--StepList:<br/>");
                while (stepList.Next())
                {
                    sb.Append("----Step: ");
                    sb.Append(getObjStr(stepList.currentStep)).Append("<br/>");
                }
                sb.Append("----leaf: ");
                sb.Append(getObjStr(stepList.LeafStep)).Append("<br/>").ToString();
            }
        }
        showText.Text = sb.ToString();
    }
    private string getObjStr(Step step)
    {
        string result = "";
        switch (step.GetBy)
        {
            case ByType.Attr:
                result = "GetBy = " + step.GetBy.ToString() + ", Name = " + step.AttrName;
                break;
            case  ByType.AttrValue:
                result = "GetBy = " + step.GetBy.ToString() + ", Name = " + step.AttrName + ", Val = " + step.AttrValue;
                break;
            case  ByType.AttrValueStart:
                result = "GetBy = " + step.GetBy.ToString() + ", Name = " + step.AttrName + ", Val = " + step.AttrValue;
                break;
            default:
                break;
        }

        return result;
    }
    /// <summary>
    /// 将分类中的每个规则rule解析为对应的StepList
    /// </summary>
    private List<StepList> ParseRules(XmlNodeList rules)
    {
        List<StepList> ruleList = new List<StepList>(rules.Count);//一个分类可能有多个解析规则
        
        foreach (XmlNode rule in rules)
        {
            XmlNodeList steps = rule.ChildNodes; //获取step和leaf 节点
            StepList stepList = ParseStepList(steps);
            ruleList.Add(stepList);
        }

        return ruleList;
    }
    private StepList ParseStepList(XmlNodeList steps)
    {
        StepList stepList = new StepList();//一个规则rule对应一个StepList        
        foreach (XmlNode s in steps)
        {
            Step step = ParseStep(s);//将xml中的每个step节点解析为Step对象
            if (s.Name == "step")
            {
                stepList.Add(step);
            }
            else if (s.Name == "leaf")
            {
                stepList.LeafStep = step;
            }
            else
            {
                //Log
            }
        }

        return stepList;
    }
    /// <summary>
    /// 将xml中的每个step节点解析为Step对象
    /// </summary>
    /// <param name="node"></param>
    /// <returns></returns>
    private Step ParseStep(XmlNode node)
    {
        string getBy = node.Attributes["getBy"].Value;
        string attrName = node.Attributes["name"].Value;
        string attrValue = node.Attributes["value"] == null ? "" : node.Attributes["value"].Value;

        Step step = new Step();
        switch (getBy)
        {
            case "Attr":
                step.GetBy = ByType.Attr;
                step.AttrName = attrName;
                break;
            case "AttrValue":
                step.GetBy = ByType.AttrValue;
                step.AttrName = attrName;
                step.AttrValue = attrValue;
                break;
            case "AttrValueStart":
                step.GetBy = ByType.AttrValueStart;
                step.AttrName = attrName;
                step.AttrValue = attrValue;
                break;
            default:
                break;
        }
        return step;
    }
    private Elements GetInitElements()
    {
        Elements eles = new Elements();
        doc = NSoup.NSoupClient.Connect("http://www.360bbk.com/").Get();
        eles = doc.GetElementsByClass("nav");
        return eles;
    }
    #endregion     
*/
    private NSoup.Nodes.Document doc = null;


    protected void ButtonTestPageParam_Click(object sender, EventArgs e)
    {
        DataRow row = Com.Toook.Common.PageData.SelectParamRow("typeName='movie2'");
        string tt = row["keyword"].ToString();
    }
}
